/*
 * AppMonitor.java
 *
 * Created on May 22, 2002, 3:56 PM
 */

package redman;

import java.net.*;
import java.util.*;
import java.io.*;
import java.text.*;
import javax.net.*;
import javax.net.ssl.*;
import org.apache.log4j.*;

/**
 *
 * @author  llaranje
 * @version 
 */
public class AppMonitor {
    
    Object handshake = new Object();
    Object monitor_sem = new Object();
    
    String name = null;
    String hamodel = null;
    String localhost = null;
    String remotehost = null;
    String allcomps = null;
    String depcomplist = null;
    String remotehosts = null;
    String initstate = "OFFLINE";
    String homedir = null;
    String startpath = null;
    String shutdownpath = null;
    String username = null;
    final String ruleofthumb = "4c6c14a5de1cacbdab5dcf71e0078c7f";
    
    Heartbeat hb = null;
    RemRedManCon remredmancon = null;
    AppMonCon appmoncon = null;
    
    int compnum = 0;
    CompMon[] compmons = null;
    int numapphosts = 0;
    String[]  apphosts = null;
    
    PrintWriter outToRemoteRedMan = null;
    PrintWriter outToRemoteAppMon = null;
    
    int numdistmngdmsgs = 0;
    int nummanaged = 0;
    int countmsgs = 0;
    
    long inittime = 0;
    long statechgtime = 0;
    long shutdowntime = 0;
    long threshold = 0, timewindow = 0, t1 = 0, t2 = 0, numfault = 0;
    
    boolean bootstart = true;
    boolean initializing = true;
    boolean starting = false;
    boolean local = false;
    boolean running = false;
    boolean startenabled = true;
    boolean failoverenabled = true;
    boolean designation = false;
    boolean waitingforhandshake = false;
    boolean switchinprogress = false;
    boolean restartinprogress = false;
    boolean newprimary = true;
    boolean badprimary = false;
    boolean reconnect = false;
    boolean firstmsgtoremote = true;
    boolean hamodupgrade = false;
    boolean activesent = false;
    
    TimerTask stateChgTask = null;
    private ComponentStartThread starter = null;
    
    private SMA engine = null;
    private HostMon hostmon = null;
    public Properties appstatictable = null;
    private Properties appdynamictable = null;
    
    private static Category trace = Category.getInstance(AppMonitor.class);
    private SocketFactory Factory = null;
   
    /** Creates new AppMonitor */
    public AppMonitor(String appname, Properties cfgtable, boolean apprestart) {       
        name = appname;
        
        trace.debug(name + ": AppMonitor (FULL) starting");
               
        // Read the home directory path information
        homedir = cfgtable.getProperty(name + "_HOMEDIR");
        if (homedir == null) {
            trace.error(name + ": home directory not defined in RedMan config file");
            return;
        }

        homedir = homedir.trim();
        
        trace.info(name + ": AppMonitor (FULL), boostart = " + bootstart); 
        
        if (!loadAndReadStaticConfigInfo()) {
            trace.error("Bad static config file. Application will not be started");
            return;
        }
        
        local = appToRunOnThisHost();
        createComponentObjects();
        
        getDependentComponentList();
        
        trace.info(name + ": nummanaged = " + String.valueOf(nummanaged));
        
        // If apprestart need to connect to remote Redman
        if (apprestart) reconnect = true;
        
        // Let things roll ...
        getGoing();
    }
    
    // Obtain list of components that depend on another component
    private void getDependentComponentList() {
        String list = " ";
        for (int i=0; i < compnum; i++) {
            String compname = compmons[i].name;
            
            // Save dependent components (if any) in the list of dependent components
            String dependents = appstatictable.getProperty(name + "_" + compname + "_DEPENDENTS");
            if (dependents != null) list += dependents + " ";           
        }
        
        depcomplist = list;
        trace.info("List of dependent components within app " + name + " = " + depcomplist);
    }

    
    public AppMonitor(String appname, boolean dummy) {
        name = appname;
        bootstart = false;
        initstate = "ACTIVE";
               
        trace.info(name + ": AppMonitor (COMPLEMENT), boostart = " + bootstart);

        // Read the home directory path information
        homedir = RedMan.cfgtable.getProperty(name + "_HOMEDIR");
        if (homedir == null) {
            trace.error(name + ": home directory not defined in RedMan config file");
            return;
        }
        
        if (!loadAndReadStaticConfigInfo()) {
            trace.error("Bad static config file. Application will not be started");
            return;
        }
        
        local = appToRunOnThisHost();
        createComponentObjects();
        getDependentComponentList();
    }
    
    public void getGoing() {        
        if (!loadAndReadDynamicConfigInfo()) {
            trace.error(name + ": bad dynamic config file - Application will not be started");
            return;            
        }        
        trace.debug(name + ": will be started, boostart = " + bootstart);
        
        // Check if app should run on this host. If so, start it (unless it is already running).
        appCheckRunAndStart();
            
        if (running) {
            if (!local) appmoncon = new AppMonCon();       
            if (local && (hamodel.equals("1+1") || hamodel.equals("1"))) 
                numdistmngdmsgs = calcDistMngdCompMsgs();
        }        
    }
    
    private boolean loadDynamicConfigFile(String filepath) {
        try {
            FileInputStream appfile = new FileInputStream(filepath);
            appdynamictable = new Properties();
            appdynamictable.load(appfile);
            return(true);
        }
        catch (FileNotFoundException e) {
            trace.error(name + " - dynamic config file not found: " + e.getMessage());
            return(false);
        }
        catch (IOException e) {
            trace.error(name + " - IOException reading dynamic config file: " + e.getMessage());
            return(false);
        }
    }
    
    private boolean loadStaticConfigFile(String filepath) {
        try {
            FileInputStream appfile = new FileInputStream(filepath);
            appstatictable = new Properties();
            appstatictable.load(appfile);
            return(true);
        }
        catch (FileNotFoundException e) {
            trace.error(name + " - static config file not found (" + filepath + "): " + e.getMessage());
            return(false);
        }
        catch (IOException e) {
            trace.error(name + " - IOException reading static config file: " + e.getMessage());
            return(false);
        }
    }
    private boolean writeDynamicConfigFile(String property, String value) {
        try {
            appdynamictable.setProperty(property, value);
            String path = homedir + "/conf/" + name + "dynamic.conf";
            OutputStream cfgout = new FileOutputStream(path);
            DateFormat time = DateFormat.getTimeInstance(DateFormat.FULL);
            appdynamictable.store(cfgout, "Dynamic configuration file for application " + name);
            trace.debug(name + ": dynamic config file successfully written");
            return(true);
        }
        catch (Exception e) {
            trace.error(name + " - Exception: " + e.getMessage());
            return(false);
        }
    }
    
    // This method loads and reads the configurable parameters that change
    // dynamically.
    private boolean loadAndReadDynamicConfigInfo() {
        String token = null;
        String path = "/home/artur/workspace/RedMan/bin/testapp/conf/"+ name +"DYNAMIC.conf";
        //String path = homedir + "/conf/" + name + "dynamic.conf";
        if (loadDynamicConfigFile(path)) {
            // Read information about start status
            token = appdynamictable.getProperty("START_ENABLED");
            if (token != null) {
                token = token.trim();
                if (token.equals("FALSE")) startenabled = false;
            }
            
            if (hamodel.equals("1")) {
                designation = true;
                // Read information about maintenance state
                token = appdynamictable.getProperty("MAINTENANCE_STATE");
                if (token != null) {
                    token = token.trim();
                    if (token.equals("TRUE")) failoverenabled = false;
                    if (token.equals("FALSE")) failoverenabled = true;
                }
            }
            else {
                // Read information about the designated primary
                token = appdynamictable.getProperty("PRIMARY");
                trace.info("PRIMARY = " + token);
                if (token != null) {
                    token = token.trim();
                    if (token.equals(localhost)) designation = true;
                    else designation = false;
                }
        
                // Read information about failover status
                token = appdynamictable.getProperty("FAILOVER_ENABLED");
                trace.info("FAILOVER_ENABLED = " + token);
                if (token != null) {
                    token = token.trim();
                    if (token.equals("FALSE")) failoverenabled = false;
                }
            }
            return(true);
        }
        else return(false);        
    }
    
    // This method loads and reads the configurable parameters that do not
    // change during the application execution without an operator command.
    private boolean loadAndReadStaticConfigInfo() {
        String token = null;        
        String path = "/home/artur/workspace/RedMan/bin/testapp/conf/"+ name +"STATIC.conf";
        //String path = homedir + "/conf/" + name + "static.conf";
        
        trace.info("Static config file path = " + path);
        
        try {
            
        if (loadStaticConfigFile(path)) {
            trace.info("Loaded static config file");
            // Read the application username information
            username = appstatictable.getProperty(name + "_USER");
            if (username != null) {
                username = username.trim();                
            }
            else {
                username = RedMan.username;
                trace.warn("No user defined for app " + name + " - Using user " + RedMan.username);
            }
            
            // Read the HA model information
            hamodel = appstatictable.getProperty(name + "_HA_MODEL");
            if (hamodel != null) {
                hamodel = hamodel.trim();                
                if (!(hamodel.equals("1+1") || hamodel.equals("1") 
                                            || hamodel.equals("1+0"))) return(false);
            }
            else {   
                trace.error("Bad config file. No HA model defined for app " + name);
                return(false);
            }
            
            trace.info("Will get information on hosts");
            if (!getInformationOnHosts()) return(false);
                             
            // Read max time for the application to change state
            if (hamodel.equals("1+1") || hamodel.equals("1")) {
                token = appstatictable.getProperty(name + "_STATE_CHANGE_TIME");
                if (token != null) {
                    token = token.trim();
                    statechgtime = Integer.parseInt(token);
                }
                else trace.warn(name + "_STATE_CHANGE_TIME not configured");
            }

            // Read the max initialization time for the application
            token = appstatictable.getProperty(name + "_INIT_TIME");
            if (token != null) {
                token = token.trim();
                inittime = Integer.parseInt(token);
                trace.info(name + "_INIT_TIME = " + token);
            }
            else trace.warn(name + "_INIT_TIME not configured");

            // Read the failure threshold for this component.
            // This is the max number of times (within the failure time window)
            // that the RedMan will cleanup and restart the application if it fails.
            String stthreshold = 
                appstatictable.getProperty(name + "_FAILURE_THRESHOLD");
            if (stthreshold != null) {
                stthreshold = stthreshold.trim();             
                threshold = Integer.parseInt(stthreshold);
            }
            // trace.debug(appname + " failure threshold = " + stthreshold);
        
            // Read the time window (to decide whether or not to restart the application)
            String sttimewindow = 
                appstatictable.getProperty(name + "_FAILURE_TIME_WINDOW");
            if (sttimewindow != null) {
                sttimewindow = sttimewindow.trim();           
                timewindow = Integer.parseInt(sttimewindow);
            }
            // trace.debug(appname + " failure time window = " + sttimewindow);
            
            // Read path for script to start the application gracefully
            token = appstatictable.getProperty(name + "_START_PATH");
            if (token != null) {
                token = token.trim();
                startpath = token;
            }
            else trace.warn(name + "_START_PATH not configured");
            
            // Read path for script to shutdown the application gracefully
            token = appstatictable.getProperty(name + "_SHUTDOWN_PATH");
            if (token != null) {
                token = token.trim();
                shutdownpath = token;
            }
            else trace.warn(name + "_SHUTDOWN_PATH not configured");

            // Read max time for the application to shutdown gracefully
            token = appstatictable.getProperty(name + "_SHUTDOWN_TIME");
            if (token != null) {
                token = token.trim();
                shutdowntime = Integer.parseInt(token);
            }
            else trace.warn(name + "_SHUTDOWN_TIME not configured");
            
            return(true);
        }
        else return(false);
        
        } // try
        catch (Exception e) {
            trace.debug("loadAndReadStaticConfigFile() Exception - " + e.getMessage());
            return(false);
        }
    }
    
    public String loadAndReadNewConfiguration() {
        String token = null;
        String oldhamodel = hamodel;
        hamodupgrade = false;
        
        trace.info("Will reload static config file");
        
        // Load new static configuration file
        String path = "/home/artur/workspace/RedMan/bin/testapp/conf/"+ name +"STATIC.conf";
        //String path = homedir + "/conf/" + name + "static.conf";
        if (loadStaticConfigFile(path)) {
            trace.info("Reloaded static config file");
            
            // Read the application user information
            username = appstatictable.getProperty(name + "_USER");
            if (username != null) {
                username = hamodel.trim();                
            }
            else {
                username = RedMan.username;
                trace.warn("No user defined for app " + name + " - Using user " + RedMan.username);
            }
            
            // Read the HA model information
            hamodel = appstatictable.getProperty(name + "_HA_MODEL");
            if (hamodel != null) {
                hamodel = hamodel.trim();                
                if (!(hamodel.equals("1+1") || hamodel.equals("1") 
                                            || hamodel.equals("1+0"))) 
                    return("Invalid HA model specified for app " + name);
            }
            else {   
                trace.error("Bad config file. No HA model defined for app " + name);
                return("No HA model defined for app " + name);
            }
                        
            // Check if upgrading from simplex ("1") to redundant ("1+1")
            trace.info("HA model = " + hamodel + ", old HA model = " + oldhamodel);
            if (hamodel.equals("1+1") && oldhamodel.equals("1")) {
                trace.info("It is an upgrade from '1' to '1+1'");
                hamodupgrade = true;
                String res = RedMan.loadConfigFile(RedMan.cfgfilepath);
                if (!res.equals("OK")) {
                    trace.info("Unable to load RedMan configuration file");
                    return("Unable to load RedMan configuration file");
                }   
                
                if (!getInformationOnHosts()) {
                    trace.info("Unable to get information on hosts");
                    return("Unable to get information on hosts");
                }
                trace.info("Will create a HostMon thread");
                hostmon = new HostMon(true);
                trace.info("Created a HostMon thread");
            }
                
            // Read max time for the application to change state for model "1+1" and "1"
            if (hamodel.equals("1+1") || hamodel.equals("1")) {
                token = appstatictable.getProperty(name + "_STATE_CHANGE_TIME");
                token = token.trim();
                statechgtime = Integer.parseInt(token);
            }

            // Read the max initialization time for the application
            token = appstatictable.getProperty(name + "_INIT_TIME");
            if (token != null) {
                token = token.trim();
                inittime = Integer.parseInt(token);
            }

            // Read max time for the application to shutdown gracefully
            token = appstatictable.getProperty(name + "_SHUTDOWN_TIME");
            token = token.trim();
            shutdowntime = Integer.parseInt(token);

            // Read the failure threshold for this component.
            // This is the max number of times (within the failure time window)
            // that the RedMan will cleanup and restart the application if it fails.
            String stthreshold = 
                appstatictable.getProperty(name + "_FAILURE_THRESHOLD");
            if (stthreshold != null) {
                stthreshold = stthreshold.trim();             
                threshold = Integer.parseInt(stthreshold);
            }
            // trace.debug(appname + " failure threshold = " + stthreshold);
        
            // Read the time window (to decide whether or not to restart the application)
            String sttimewindow = 
                appstatictable.getProperty(name + "_FAILURE_TIME_WINDOW");
            if (sttimewindow != null) {
                sttimewindow = sttimewindow.trim();           
                timewindow = Integer.parseInt(sttimewindow);
            }
            // trace.debug(appname + " failure time window = " + sttimewindow);
                        
            // Read the new dynamic configuration information
            if (loadAndReadDynamicConfigInfo()) {
                // If HA model being upgraded inform application processes
                if (hamodupgrade) {
                    engine.setDesignation(designation);
                    trace.info("Due to HA model upgrade will send config change HA_MODEL to app " + name);
                    notifyAppOfConfigChange("HA_MODEL");
                    trace.info("Config change HA_MODEL sent to app " + name);
                }
                return("OK");
            }
            else {
                return("Unable to load or read dynamic config file");
            }
        }
        else return("Unable to load or read static config file");
    }
    
    private boolean appCheckRunAndStart() {
        trace.debug("Starting appCheckRunAndStart()");
        if (local) {
            trace.debug(name + ": is supposed to run on this host");
            if (!hamodel.equals("1") && (hostmon == null)) hostmon = new HostMon(false); 
            if (reconnect) startThreadToConnectToRemApp();
            trace.debug(name + ": boostart = " + bootstart + ", hamodel = " + hamodel);

            if (hamodel.equals("1+1") || hamodel.equals("1")) {
                // Create state model object according to RedMan start/restart status
                if (engine == null) engine = new SMA(name, hamodel, initstate, designation, failoverenabled);
                     
                // Start application if start enabled
                if (startenabled) {
                    running = true;
                    appStart(name, bootstart);
                    trace.debug(name + ": locally started on this host (" + localhost + ")");
                    if (!bootstart) {
                        trace.debug(name 
                            + ": assumed to be already running on this host (" + localhost + ")");
                    }
                }
            }
            if (hamodel.equals("1+0")) {
                String state;
                // Create state model object according to RedMan start/restart status
                if (bootstart) {
                    if (engine == null) {
                        engine = new SMA(name, hamodel, "OFFLINE", designation, failoverenabled);
                        engine.run("SMA INITIALIZE");
                    }
                                        
                    // Start application if start enabled
                    if (startenabled) {
                        running = true;
                        appStart(name, bootstart);
                        trace.debug(name + ": locally running on this host (" 
                                            + localhost + ")");
                        return(true);
                    }
                    else {
                        trace.debug(name + ": not running on this host (" 
                                                + localhost + ")");
                    }
                }
                else if (engine == null) {
                        engine = new SMA(name, hamodel, initstate, designation, failoverenabled);
                        if (initstate.equals("ACTIVE")) {
                            trace.debug(name + ": should already be running on this host (" 
                                                    + localhost + ")");
                        }
                }
            }
            return(true);
        }
        else if (distCompToRunOnThisHost()) {
            local = false;
            if (hamodel.equals("1+1") || hamodel.equals("1")) {
                if (startenabled) {
                    running = true;
                    appStart(name, bootstart);
                    trace.debug(name + ": started on this host (" 
                                            + localhost + ") running distributed");                    
                }
            }
            if (hamodel.equals("1+0")) {
                if (engine.getState().equals("ACTIVE")) {
                    if (startenabled) {
                        running = true;
                        appStart(name, bootstart);
                        trace.debug(name + ": started on this host (" 
                                                + localhost + ") running distributed");                        
                    }
                }
            }
            return(true);
        }
        else {
            trace.debug(name + ": is not supposed to run on this host (" 
                                    + localhost + ")");
            return(false);
        }
    }
        
    private boolean appToRunOnThisHost() {
        boolean run = false;
        
        try {            
            String hosts = appstatictable.getProperty(name + "_HOSTS");
            StringTokenizer st = new StringTokenizer(hosts);
            
            trace.debug(name + ": hosts = " + hosts + ", localhost = " + localhost);
       
            if ((numapphosts = st.countTokens()) <= 0) {
                trace.error(name + ": Bad config file. No host names in application definition");
                return(false);
            }
       
            apphosts = new String[numapphosts];
            
            for (int i = 0; i < numapphosts; i++) {
                apphosts[i] = st.nextToken();
                if (apphosts[i].equals(localhost)) run = true;
            }
            
            return(run);
        }
        catch (Exception e) {
            trace.error(name + " - Exception: " + e.getMessage());
            System.exit(2);
        }
        
        return(run);
    }
    
    private boolean distCompToRunOnThisHost() {
        String compname = null;
        String stlocality = null;
        String hosts = null;
        String hostname = null;
        boolean run = false;
        try {            
            StringTokenizer st = new StringTokenizer(allcomps);
            
            while (st.hasMoreTokens()) {
                compname = st.nextToken();
                stlocality = appstatictable.getProperty(name + "_" + compname + "_LOCALITY");
                if (stlocality != null) stlocality = stlocality.trim();
                if (stlocality.equals("REMOTE")) {
                    hosts = appstatictable.getProperty(name + "_" + compname + "_HOSTS");
                    if (hosts == null) {
                        trace.error(name 
                            + ": Bad config file. No host names defined for remote component " 
                            + compname);
                        break;
                    }
                    StringTokenizer ht = new StringTokenizer(hosts);
                    while (ht.hasMoreTokens()) {
                        hostname = ht.nextToken();
                        if (hostname.equals(localhost)) {
                            run = true;
                            break;
                        }
                    }
                }
            }
            
            return(run);
        }
        catch (Exception e) {
            trace.error(name + " - Exception: " + e.getMessage());
            System.exit(2);
        }
        
        return(run);
    }
    
    // This method calculates the number of remote  messages  this
    // application monitor will receive due to distributed managed
    // components. This method assumes the application  is  local,
    // that is, its state machine runs on the local host.
    private int calcDistMngdCompMsgs() {
        int nummsgs = 0;
        
        // Calculate the index of the application monitor host
        int index = 0;
        for (int i=0; i < numapphosts; i++) {
            if (localhost.equals(apphosts[i])) {
                index = i;
                break;
            }
        }
              
        String compname1, compname2;
        String hosts, host1=null, host2=null;
        StringTokenizer st1, st2, stokhosts;
        
        st1 = new StringTokenizer(allcomps);
        
        for (int i=0; i < compnum; i++) {  // Iterate over all components
            // Get next component
            compname1 = st1.nextToken();
            
            // Check if component is remote and managed
            if (isDistAndManaged(compname1)) {
                // Get the host where this remote managed component runs
                hosts = appstatictable.getProperty(name + "_" + compname1 + "_HOSTS");
                stokhosts = new StringTokenizer(hosts);
                for (int j=0; j < index; j++) host1 = stokhosts.nextToken();
                
                // Iterate over remainder components to check  if  there  is
                // another distributed managed component running on the same
                // host host1.
                st2 = st1;
                boolean found = false;
                while (st2.hasMoreTokens()) {
                    compname2 = st2.nextToken();
                    if (isDistAndManaged(compname2)) {
                        hosts = appstatictable.getProperty(name + "_" + compname2 + "_HOSTS");
                        stokhosts = new StringTokenizer(hosts);
                        for (int j=0; j < index; j++) host2 = stokhosts.nextToken();
                    }
                    if (host1.equals(host2)) {
                        found = true;
                        break;
                    }
                }
                
                if (!found) nummsgs++;
            }
        }
        return(nummsgs);
    }
    
    private boolean isDistAndManaged(String compname) {
        String stlocality, stmsgs, msg, subcomps, subcomp;
        boolean statemsg = false;
        boolean remstatemsg = false;
            
        // Check if component is distributed
        stlocality = appstatictable.getProperty(name + "_" + compname + "_LOCALITY");
        if (stlocality != null) {
            stlocality = stlocality.trim();
            if (!stlocality.equals("DISTRIBUTED")) return(false); // Component is local
        }
        else return(false); // Component is local by default
                
        // Read type information (DAEMON, PROCESS or PROCESS_GROUP)
        String type = appstatictable.getProperty(name + "_" + compname);
        if (type != null) type = type.trim();
        else return(false); // component type not specified, igonored
        trace.debug("Distributed component " + name + ":" + compname + " type is " + type);
            
        // Check if component is managed
            
        stmsgs = appstatictable.getProperty(name + "_" + compname + "_MESSAGES");
        if (stmsgs != null) {
            StringTokenizer st = new StringTokenizer(stmsgs);
            while (st.hasMoreTokens()) {
                msg = st.nextToken();
                if (msg.equals("STATE")) statemsg = true;
                if (msg.equals("REMOTE_STATE")) remstatemsg = true;
            }
            if  (statemsg) return(true);
            else if (!type.equals("PROCESS_GROUP")) return(false);  // Not managed
        }
        else if (!type.equals("PROCESS_GROUP")) return(false); // Not managed by default
            
        // Here we have a PROCESS_GROUP for which managed status not was specified at
        // component level. We need to check if at least one subcomponent is managed.
 
        subcomps = appstatictable.getProperty(name + "_" + compname + "_SUBCOMPONENTS");
        if (subcomps == null) return(false); // No subcomponents, ignored
        StringTokenizer st1 = new StringTokenizer(subcomps);
        while (st1.hasMoreTokens()) {
            subcomp = st1.nextToken();
            stmsgs = appstatictable.getProperty(name + "_" + compname + "_" + subcomp + "_MESSAGES");
            if (stmsgs != null) {
                StringTokenizer st2 = new StringTokenizer(stmsgs);
                while (st2.hasMoreTokens()) {
                    msg = st2.nextToken();
                    if (msg.equals("STATE")) statemsg = true;
                    if (msg.equals("REMOTE_STATE")) remstatemsg = true;
                }
                if (statemsg) return(true);  // Found one managed subcomponent
            }
        }
        return(false);   // Found no managed subcomponents.
    }
    
    public boolean checkDistHost(String host, PrintWriter out) {
        if (!local) return(false);
        boolean retval = false;
        for (int i = 0; i < compnum; i++) {
            if (!compmons[i].local && compmons[i].host.equals(host)) {
                retval = true;
                compmons[i].out = out;
            }
        }
        return(retval);
    }
    
    private boolean getInformationOnHosts() {
        String hostname = null;
        String hostlist = null;
        boolean foundlocalhost = false;
        
        try {
            // Read information about hosts
            localhost = InetAddress.getLocalHost().getHostName();
        }
        catch (UnknownHostException e) {
            trace.error(name + " - Exception: " + e.getMessage());
            return(false);
        }
        
        try {
            String hosts = appstatictable.getProperty(name + "_HOSTS");
            trace.debug(name + ": hosts = " + hosts + ", localhost = " + localhost);
            
            StringTokenizer st = new StringTokenizer(hosts);
       
            // Check if list of hosts in the config file is empty
            if (st.hasMoreTokens() == false) {
                trace.debug(name + 
                            ": Bad config file. No host names defined for application ");
                return(false);
            }
       
            if (st.countTokens() == 1) {
                hostname = st.nextToken();
                if (!hostname.equals(localhost)) return(false);
                else return(true);
            }
            
            while (st.hasMoreTokens()) {
                hostname = st.nextToken();
                if (!(hostname.equals(localhost))) {
                    if (hostlist == null) hostlist = hostname;
                    else hostlist += " " + hostname;
                    remotehost = hostname;
                }
                else foundlocalhost = true;
            }
            trace.debug(name + " - List of remote hosts: " + hostlist);
            remotehosts = hostlist;
            if ((hamodel.equals("1+1") || hamodel.equals("1+0")) && 
                            ((remotehost == null) || (hostlist == null))) {
                trace.error(name + " - Bad config file - No remote host specified for hamodel " + hamodel);
                return(false);
            }
            if (!foundlocalhost) {
                trace.error(name + " - Bad config file - Local host not specified in config file");
                return(false);
            }
            return(true);
        }
        catch (Exception e) {
            trace.error(name + " - Exception: " + e.getMessage());
            return(false);
        }    
    }
    
    public boolean isInList(String item, String list) {
        StringTokenizer st = new StringTokenizer(list);
        String token;
        if (!st.hasMoreTokens()) return(false);
        else {
            while (st.hasMoreTokens()) {
                token = st.nextToken();
                if (item.equals(token)) return(true);
            }
            return(false);
        }
    }
    
    public void startThreadToConnectToRemApp() {
        if (outToRemoteRedMan == null) {
            if (remredmancon == null) remredmancon = new RemRedManCon();
            else trace.debug("startThreadToConnectToRemApp() not starting a new thread - one already exists");
        }
        else trace.debug("startThreadToConnectToRemApp() not starting a new thread - connection exists");
    }
    
    private void handleHostEvent() {
        //trace.debug("handleHostEvent() called, event=\"" 
        //                      + hb.event + "\", remotehosts=\"" + remotehosts + "\"");
        StringTokenizer st = new StringTokenizer(hb.event);
        String host = st.nextToken();
        String ev = st.nextToken();
        
        trace.info(name + ": received hostevent " + ev + " from host " + host);
        
        if (isInList(host, remotehosts)) {
            if (ev.equals("DOWN")) {
                trace.debug(name + ": remote host " + host + " is down.");
                outToRemoteRedMan = null;
                // remredmancon = null;
                String newstate = submitEventToSMA("RHOST DOWN");
                notifyAppOfRemStateChange(" FAILED COMM");
                if (hamodel.equals("1+0") && newstate.equals("ACTIVE")) {
                    appStart(name, true);
                }
            }
            else if (ev.equals("UP")) {
                trace.debug(name + ": remote host " + host + " is up");
                trace.debug(name + ": handleHostEvent() will call startThreadToConnectToRemApp()");
                startThreadToConnectToRemApp();
                //if (!bootstart) newprimary = true;               
                
            }
        }
    }
    
    private class ComponentStartThread extends Thread {
        private boolean stopped = false;
        private boolean script;
        Process p = null;
        
        public void cancel() {
            if ((script) && (p != null)) p.destroy();
            stopped = true;
        }
        
        public ComponentStartThread (boolean script) {
            this.script = script;
            start();
        }
        
        private void startAppComp(int ind) {   
            // Start component
            trace.info("startAppComp() will start component " + name + ":" + compmons[ind].name);            
            compmons[ind].startComponent();
        
            // Find if there are dependent components and start them
            String dependents = compmons[ind].dependentcomps;
            trace.info(name + ":" + compmons[ind].name + " DEPENDENTS = " + dependents);
            if (dependents != null) {
                trace.info(name + ":" + compmons[ind].name + " DEPENDENTS = " + dependents);
                StringTokenizer st = new StringTokenizer(dependents);         
                while (st.hasMoreTokens()) {
                    String compname = st.nextToken();
                    int index = findCompObjIndex(compname);
                    if (index < 0) {
                        trace.error("Dependent component '" + name + ":" + compname
                                                            + "' not found - Will not be started");
                    }
                    else {
                        trace.info("startAppComp() will start dependent component " + name + ":" + compname);
                        compmons[index].startComponent();
                    }
                }
            }
        }
        
        private void getPidsFromStartScritp() {
            String stpid, pidstring;
            boolean pidsok;
            InputStream pin;
            InputStreamReader cin;
            BufferedReader in;
            try {                
                // Get pids of application processes
                pin = p.getInputStream();
                cin = new InputStreamReader(pin);
                in = new BufferedReader(cin);
            }
            catch (Exception e) {
                trace.error(name + ": failed to get input stream from application start script");
                return;
            }
            
            for (int i=0; i < compnum; i++) {
                if (stopped) break;
                pidstring = " ";
                pidsok = true;
                if (compmons[i].type.equals("PROCESS_GROUP")) {
                    pidsok = true;
                    // Get pids for process group processes
                    for (int j=0; j < compmons[i].subcompnum; j++) { 
                        if (stopped) break;                            
                        try {
                            stpid = in.readLine();
                            if (stpid != null) {
                                stpid = stpid.trim();
                                int pid = Integer.parseInt(stpid);
                                pidstring += stpid + " ";                
                                String procname = name + ":" + compmons[i].name + ":" 
                                                       + compmons[i].subcompmons[j].pname;
                                trace.info("Got pid for process " + procname + ", PID = " + stpid);
                                CompMon.renicePid(procname, pid);
                                compmons[i].subcompmons[j].subcomppid = pid;
                            }
                            else pidsok = false;
                        }
                        catch (Exception e) {
                            trace.error(name + " - Exception: " + e.getMessage() 
                                             + " - Failed to get pid for component process '" 
                                             + name + ":" + compmons[i].name + "'");
                            pidsok = false;                            
                        }
                    }                   
                }
                else {
                    pidsok = true;
                    try {
                        stpid = in.readLine();
                        if (stpid != null) {
                            stpid = stpid.trim();
                            pidstring += stpid + " ";
                            int pid = Integer.parseInt(stpid);
                            String procname = name + ":" + compmons[i].name;
                            trace.info("Got pid for process " + procname + ", PID = " + stpid);
                            CompMon.renicePid(procname, pid);
                            compmons[i].comppid = pid;
                        }
                        else pidsok = false;
                    }
                    catch (Exception e) {
                        trace.error(name + " - Exception: " + e.getMessage() 
                                         + " - Failed to get pid for component process '" 
                                         + name + ":" + compmons[i].name + "'");
                        pidsok = false;
                    }                    
                }
                if (pidsok) compmons[i].sendPidStringToBackupRedMan(pidstring);
            }
        }
        
        // Execute script that starts the entire application.
        // Read from the script all pids of the application processes.
        // Send these pids to the backup RedMan.
        private Process appOneScriptStart() {
            Process p;
            try {
                trace.info("RedMan will execute script " + startpath + " to start entire app " + name);
                p = Runtime.getRuntime().exec(startpath);
            }
            catch (Exception e) {
                trace.error(name + " - Exception: " + e.getMessage() 
                                 + " - Failed to execute application script");
                return null;
            }
            return p;
        }
        
        public void run() {
            int i;
            starting = true;
            if (script) {
                p = appOneScriptStart();
                try {
                    p.wait();
                    getPidsFromStartScritp();
                    if (stopped) {
                        starting = false;
                        starter = null;
                        return;
                    }
                }
                catch (Exception e) {
                    starting = false;
                    starter = null;
                    trace.debug("Process starting app " + name + " was interrupted or killed");
                    return;
                }
            }
            else {
                for (i=0; i < compnum; i++) {
                    if (stopped) {
                        starting = false;
                        starter = null;
                        return;
                    }
                    String compname = compmons[i].name;
                    if (isInList(compname, depcomplist)) continue;
                        trace.info("Start of component " + name + ":" + compname 
                                        + " and its dependent components (if any) will be attempted");
                    startAppComp(i);
                }
            }
            
            starting = false;
            
            // Check if all components are started (have a valid pid)            
            if (allAppPidsValid()) {
                trace.debug("App new pids are valid - Will start monitoring threads for app " + name);
                kickMonitoringThreads();
                starter = null;
            }
            else {   
                starter = null;
                trace.error(name + ": application startup failed");
                handleFailure("FLT");
            }
        }
    }
    
    private boolean allAppPidsValid() {
        boolean valid = true;;
        for (int i=0; i < compnum; i++) {
            if (compmons[i].type.equals("PROCESS_GROUP")) {
                for (int j=0; j < compmons[i].subcompnum; j++) {
                    if (compmons[i].subcompmons[j].subcomppid <= 0) {
                        trace.debug("Pid for process " + name + ":" + compmons[i].name 
                                         + ":" + compmons[i].subcompmons[j].pname + " found invalid");
                        valid = false;
                        break;
                    }
                }
                if (!valid) break;
            }
            else {
                if (compmons[i].comppid <= 0) {
                    trace.debug("Pid for process " + name + ":" + compmons[i].name + " found invalid");
                    valid = false;
                    break;
                }
            }
        }
        return(valid);
    }
    
    public int findCompObjIndex(String comp) {
        int i, index = -1;
        for (i = 0; i < compnum; i++) {
            if (comp.equals(compmons[i].name)) {
                index = i;
                break;
            }
        }
        return(index);
    }
    
    public void assignComponentPids(String compname, String pidstring) {
        int index = -1;
        
        // Find the component object
        for (int i=0; i < compnum; i++) {
            trace.debug("compmons[i].name = " + compmons[i].name + ", i=" + String.valueOf(i));
            if (compname.equals(compmons[i].name)) {
                index = i;
                break;
            }
        }
        if (index < 0) {
            trace.error("Component not found (" + compname + ") for pid message from Primary RedMan");
            return;
        }
        else {
            trace.debug("Component found (" + compname + "), index = " + String.valueOf(index) 
                                            + ", pidstring = " + pidstring);
        }
        compmons[index].assignPids(pidstring);
    }
    
    private void createComponentObjects() {
        // Get number and list of all components
        allcomps = appstatictable.getProperty(name + "_COMPONENTS");
        trace.info("Components of app " + name + ": " + allcomps);
        StringTokenizer st = new StringTokenizer(allcomps);
        if ((compnum = st.countTokens()) <= 0) {
            trace.error(name + 
                    ": Bad config file. No component names in application definition.");
            return;  // Will not start any components
        }
        
        // Initialize array of component monitors
        compmons = new CompMon[compnum];
        
        // Create component monitor objects for this application
        int i = 0;
        while (st.hasMoreTokens()) {
            String compname = st.nextToken();
            compmons[i] = new CompMon(name, this, compname);
            i++;
        }
    }
    
    private void kickMonitoringThreads() {
        boolean threadsAlive = true;
        for (int i=0; i < compnum; i++) {
            if (!compmons[i].isAlive()) {
                threadsAlive = false;
                break;
            }
        }
        
        if (threadsAlive) {
            try {
                synchronized (monitor_sem) {
                    trace.info("Will notify monitoring threads for app " + name + " to loop around");
                    monitor_sem.notifyAll();
                }
            }
            catch (Exception e) {
                trace.warn(name + " - monitor_sem.notifyAll() exception:" + e.getMessage());
            }  
            return;
        }
        
        // Start threads that monitor the application components
        for (int i=0; i < compnum; i++) {
            trace.info("Will start the monitoring thread for component " 
                                       + name + ":" + compmons[i].name);            
            compmons[i].start();
        }        
    }
         
    
    private void appStart(String appname, boolean bootstart) {
        int i;
        boolean startcomponents = bootstart;
        nummanaged += numdistmngdmsgs;  // Potential Change
        countmsgs = 0;
        initializing = true;
        
        if (bootstart) {
            // Submit the appropriate event to the state machine
            if (hamodel.equals("1+1") || hamodel.equals("1")) {
                // Set state machine to state INIT   
                submitEventToSMA("APP STARTED");
                commitNewSMAState();
            }     
            if (hamodel.equals("1+0")) {
                submitEventToSMA("APP STARTED");
                sendStateToRemoteRedMan("ACTIVE");
            }
        }
        
        // Check if there is a start script that starts the whole application
        if ((startpath != null) && bootstart) {
            startcomponents = false;
            starter = new ComponentStartThread(true);
        }
        
        // Start lead components (those who are not dependent on another 
        // component) one by one if a script to start the entire
        // application was not specified
        if (startcomponents) {
            starter = new ComponentStartThread(false);
        }        
        else {  
            if (allAppPidsValid()) kickMonitoringThreads();
            else {
                if ((startpath != null) && bootstart) {
                    trace.error(name + ": application startup (one script) failed");
                    handleFailure("FLT");
                }
                else {
                    trace.error(name + " failed" 
                                 + " - RedMan did not have valid pids for app processes");
                    handleFailure("FLT");
                }
            }
        }
    }
    
    private class HostMon extends java.lang.Thread {
        boolean upgrade;
        boolean created = false;
        
        Heartbeat loadAndCreateHeartbeat(String myhb) {
            String hblist = RedMan.cfgtable.getProperty("HEARTBEATS");
            trace.debug("RedMan HEARTBEATS = " + hblist);

            StringTokenizer st = new StringTokenizer(hblist);
        
            while (st.hasMoreTokens()) {
                String hbname = st.nextToken();
                if (myhb.equals(hbname)) {
                    try {
                        RedMan.hbs[RedMan.hbnum] = new Heartbeat(myhb);
                    }
                    catch (Exception e) {
                        trace.error("Exception while creating new heartbeat object: " + e.getMessage());
                        return(null);
                    }
                    RedMan.hbnum++;
                    return(RedMan.hbs[RedMan.hbnum-1]);    
                }
            }
            return(null);
        }
        
        private Heartbeat findHeartbeat(boolean upgrade) {
            Heartbeat foundhb = null;
            int index = -1;
            String myhb = appstatictable.getProperty(name + "_HEARTBEAT");
            if (myhb != null) myhb = myhb.trim();
            else return(null);
            int i;
            for (i=0; i < RedMan.hbnum; i++) {
                if (RedMan.hbs[i].name.equals(myhb)) {
                    foundhb = RedMan.hbs[i];
                    index = i;
                }
            }
            if (upgrade) {
                if (foundhb == null)  {
                    foundhb = loadAndCreateHeartbeat(myhb);
                    if (foundhb != null) created = true;
                }
/**********************
                else {
                    trace.info("Will destroy old heartbeat threads");
                    foundhb.cleanup();
                    trace.info("Destroyed old heartbeat threads");
                    try {
                        RedMan.hbs[index] = new Heartbeat(myhb);
                        trace.info("Created new heartbeat " + myhb);
                        foundhb = RedMan.hbs[index];
                    }
                    catch (Exception e) {
                        trace.error("Exception creating heartbeat " + myhb + " (HA model upgrade)");
                        RedMan.hbs[index] = null;
                        foundhb = null;
                    }
                }
 ********************/
            }            
            return (foundhb);
        }
            
    
        /** Creates new HostMon object */
        public HostMon(boolean upgrade) {
            this.upgrade = upgrade;
            hb = findHeartbeat(upgrade);
            if (hb == null) {
                trace.error(name + ": Bad config file. Application heartbeat not defined.");
                return;
            }
            
            // Only start the HbMon thread if the heartbeat defined for this
            // application is supposed to run on the local host.
            if (hb.torun) start();
            else {
                trace.error(name + ": heartbeat for application not running on local host");
            }
            
            // if (upgrade) hb.startHeartbeat();
        }

        public void run() {     // HostMon
            boolean doitonce = false;
            if (created) doitonce = true;
            startThreadToConnectToRemApp();
            trace.info(name + ": HostMon thread is running.");
            while (true) {
                try {
                    trace.debug("HostMon (app = " + name +
                                         ") will synchronize on semaphore");
                    synchronized (hb.sem) {
                        if (doitonce) {
                            trace.info("Will start heartbeat");
                            hb.startHeartbeat();
                            doitonce = false;
                        }
                        trace.debug("HostMon (app = " + name +
                                         ") will wait for hostevent");
                        hb.sem.wait();
                        trace.debug("HostMon (app = " + name +
                                         ") received hostevent - Will call handleHostEvent()");
                    }                   
                    handleHostEvent();
                }
                catch (Exception e) {
                    if (hostmon == null) break;
                    trace.error(name + " - Exception:" + e.getMessage());                    
                }
            }
            trace.debug("HostMon thread exiting ...");
        }
    }
    
    private void stopHostMon() {
        HostMon tobestopped = hostmon;
        hostmon = null;
        if (tobestopped != null) {
            tobestopped.interrupt();
            trace.debug(name + ": hostmon thread stopped");
        }
        else trace.debug("hostmon thread could not be stopped - Thread object was null");
    }
    
    public void sendAppPidsToBackupRedMan() {
        for (int i = 0; i < compnum; i++) {
            trace.debug("Will send pid(s) of component " + compmons[i].name + " to backup RedMan");
            compmons[i].sendCompPidsToBackupRedMan();
        }
    }
    
    public void resolvePrimaryDesignation() {
        String nprimary;
        if (localhost.compareTo(remotehost) < 0) nprimary = localhost;
        else nprimary = remotehost;
        String state = getSMAState();
        if (!initializing && !state.equals("OFFLINE") && !state.equals("FAILED"))
            setNewPrimaryHost(nprimary, true);
        else {
            setNewPrimaryHost(nprimary, false);
            badprimary = true;
        }
        trace.info(name + ": on host " + localhost + " primary being resolved to " + remotehost); 
    }

    public void resolveFailoverEnabledConfiguration() {
        if (!designation) {
            boolean changeto = !failoverenabled;
            trace.debug("Local host is not primary for app " + name 
                                    + ": will change failover enabled configuration to " + changeto);
            if (failoverenabled) inhibitFailoverLocal();
            else uninhibitFailoverLocal();
        }
        else trace.debug("Local host is primary for app " + name 
                                    + ": no need to change failover enabled configuration ("
                                    + failoverenabled + ")");
    }
    
    private void sendStateToRemoteRedMan(String state) {
        try {
            if (outToRemoteRedMan != null) {
                String msg = "RAPP STATE " + name + " " + localhost + " " + state;
                
                if (firstmsgtoremote) {
                    // Send primary designation
                    if (designation) msg += " PRIMARY";
                    else msg += " SECONDARY";
                    
                    // Send failover enabled configuration
                    if (failoverenabled) msg += " UNINHIBITED";
                    else msg += " INHIBITED";
                    firstmsgtoremote = false;
                }
                
                //
                // Check if :
                //
                // (a) This is the first time a new primary RedMan is  sending
                //     state to the remote application. If it is, signal it to
                //     the remote RedMan, so it will also connect to this 
                //     RedMan (which is a new primary).
                //
                // (b) The application has been restarted
                //
                // (c) There has been an upgrade in the ha model from 1 to 1+1
                //
                // If any of the above conditions is true, insert token in the remote
                // state message ("NEW") that instructs the remote RedMan to reconnect
                // to this RedMan
                //
                if ((newprimary) || (reconnect && !state.equals("OFFLINE")) || hamodupgrade) {
                    msg += " NEW";
                    newprimary = false;
                    reconnect = false;
                }
                
                outToRemoteRedMan.println(msg);
                outToRemoteRedMan.flush();
                
                if (state.equals("ACTIVE")) activesent = true;
                
                trace.info(name + ": RedMan sending msg \"" + msg + "\"");
            }
            else {
                if (!hamodel.equals("1"))
                    trace.warn(name + ": output stream to remote application monitor is null");
            }
        }
        catch (Exception e) {
            trace.error(name + " - Exception: " + e.getMessage());
        }
    }
    
    public void sendCommandToRemoteRedMan(String cmd) {
        try {
            if (outToRemoteRedMan != null) {                    
                String msg = "RAPP CMD " + name + " " + localhost + " " + cmd;
                                
                // Check if this is the first time a new primary RedMan is sending
                // a command/state to the remote application. If it is, signal
                // it to the remote RedMan, so it will also connect to this 
                // RedMan (which is a new primary).
                if (newprimary) {
                    msg = msg + " NEW";
                    newprimary = false;
                }
                
                outToRemoteRedMan.println(msg);
                outToRemoteRedMan.flush();
                
                trace.info(name + ": RedMan sending msg \"" + msg + "\"");
            }
            else {
                trace.warn(name + ": output stream to remote application monitor is null");
            }
        }
        catch (Exception e) {
            trace.error(name + " - Exception: " + e.getMessage());
        }
    }
    
     private boolean checkWithRedMan(BufferedReader in, PrintWriter out) {
        try {              
            if (out != null) {
                String msg = "REDMAN " + ruleofthumb;
                             
                // System.out.println("opint: user/passwd to RedMan = " + msg); 
                out.println(msg);
                out.flush();
                
            }
            else {
                trace.error("Not connected to RedMan");
                return(false);
            }
            String resp = in.readLine();
            if ((resp == null) || (resp.length() == 0) || !resp.equals("ACK")) {
                trace.error("Authentication failed - resp = " + resp); 
                return(false);
            }
            else {
                // trace.info("Authentication suceeded");
                return(true);
            }
        }
        catch (Exception e) {
            trace.error("Exception: " + e.getMessage());
            return(false);
        }
    }   
    
    // This class specifies a thread that connects to the  remote  RedMan
    // running the application monitor state machine that manages dist.
    // app components of this application that run on the local host (if 
    // they exist). Once the connection is established this thread receives 
    // messages from the remote app monitor and acts upon  them  sending  
    // the  corresponding  messages  to  the  remote components runing on 
    // the local host (if any).
    public class AppMonCon extends java.lang.Thread {
        String appmonhost = null;
        BufferedReader inFromRemoteAppMon = null;
        PrintWriter outToRemoteAppMon = null;
        
        /** Creates new AppMonCon object */
        public AppMonCon() {                     
            // Find in which host is the application monitor for this dist app running
            appmonhost = findAppMonHost();
            
            trace.debug(name 
                + ": remote app monitor for dist components on host " 
                + localhost + " is running on host " + appmonhost);
            
            setPriority(Thread.currentThread().getPriority() + 1);
            if (appmonhost != null) start();
        }
        
        private String findAppMonHost() {
            int numcomps = 0;
            int numcomphosts = 0;
            int numapphosts = 0;
            int numhosts = 0;
            String compname = null;
            String comphosts = null;
            String hostname = null;
            StringTokenizer st = null;
            StringTokenizer ht = null;

            try {            
                String comps = appstatictable.getProperty(name + "_COMPONENTS");
                st = new StringTokenizer(comps);
       
                if ((numcomps = st.countTokens()) <= 0) {
                    trace.error(name + 
                        ": Bad config file. No component names in application definition.");
                    return(null);
                }
                
                while (st.hasMoreTokens()) {
                    compname = st.nextToken();
                    comphosts = appstatictable.getProperty(name + "_" + compname + "_HOSTS");
                    if (comphosts == null) {
                        trace.error(name 
                            + ": Bad config file. No host names defined for remote component " 
                            + compname);
                        return(null);
                    }
                    ht = new StringTokenizer(comphosts);
                    numcomphosts = ht.countTokens();
                    if (numcomphosts <= numapphosts) numhosts = numcomphosts;
                    else numhosts = numapphosts;
                    
                    for (int i = 0; i < numhosts; i++) {
                        hostname = ht.nextToken();
                        if (hostname.equals(localhost)) return(apphosts[i]);
                    }
                }
            }
            catch (Exception e) {
                trace.error(name + " - Exception: " + e.getMessage());
                System.exit(2);
            }
        
            return(null);
        }
        
        public BufferedReader connectToRemoteAppMon(String hostname) {
            try {
                if (Factory == null )
                {
                    // Get the secure socket factory
                    trace.info("Will obtain the default SSL socket factory");
                    Factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
                    trace.info("Obtained the default SSL socket factory");
                }
                
                // Connect to remote RedMan
                //Socket so = new Socket(hostname, RedMan.port);
                SSLSocket so = (SSLSocket) Factory.createSocket(hostname, RedMan.port);
                
                so.setTcpNoDelay(true);
            
                // Get I/O streams to talk to the app monitor RedMan                
                inFromRemoteAppMon  = new BufferedReader(new InputStreamReader(so.getInputStream()));
                outToRemoteAppMon = new PrintWriter(new OutputStreamWriter(so.getOutputStream()));
                if (checkWithRedMan(inFromRemoteAppMon, outToRemoteAppMon)) return(inFromRemoteAppMon);
                else {
                    trace.info("Authentication with remote AppMon failed");
                    return(null);
                }
            }
            catch (Exception e) {
                trace.error(name + ": - Exception: " + e.getMessage());
                return(null);
            }
        }
            
        private void getMsgsFromRemoteAppMon() {
            try {
                for (String msg = null; (msg = inFromRemoteAppMon.readLine()) != null;) {
                    if (msg.length() == 0) break;
                    msg = msg.trim();
                    trace.debug(name + ": dist app received msg: \"" + msg + "\" from remote app monitor");
                }
            }
            catch (Exception e) {
                trace.error(name + " - Exception: " + e.getMessage());
                e.printStackTrace();
            }
        }
        
        
        public void run() {     // AppMonCon
            trace.debug(name 
                + ": AppMonCon connection thread is running on this host (" 
                + localhost + ")");
            
            // Attemp to connect
            for (int i = 1; i <= 10; i++ ) {
                try {
                    // Connect to remote application monitor RedMan
                    inFromRemoteAppMon = connectToRemoteAppMon(appmonhost);
                    if (inFromRemoteAppMon != null) {                        
                        trace.info(name + "(distributed app on host " + localhost 
                                        + "): successfully connected to its application monitor on host "
                                        + appmonhost);
                        break;
                    }
                    sleep(1000);
                }
                catch (Exception e) {
                    trace.debug(name + " - Exception: " + e.getMessage());
                }
            }
            
            // Receive and process messages from the remote application monitor
            if (inFromRemoteAppMon != null) getMsgsFromRemoteAppMon();
        } 
    }
    
    private class RemRedManCon extends java.lang.Thread {
        BufferedReader fromRedMan = null;
        PrintWriter toRedMan = null;
        
        /** Creates new RemRedManCon object */
        public RemRedManCon() {
                       
            // Only start the RemRedManCon thread if an instance of the application
            // is configured to run on a distributed host.
            if (remotehosts == null) {
                trace.warn(name + ": remote instance of application not configured.");
            }
            else {
                trace.debug(name + " - app remote hosts: " + remotehosts);
                setPriority(Thread.MAX_PRIORITY - 1);
                start();
            }
        }
        
        public PrintWriter connectToRemoteRedMan(String hostname) {
            try {
                if (Factory == null)
                {
                    trace.info("Will obtain the default SSL socket factory");
                    // Get the secure socket factory
                    Factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
                    trace.info("Obtained the default SSL socket factory");
                }

                // Connect to remote RedMan
                trace.info(name + ": will connect to remote RedMan on host " + hostname);
                //Socket so = new Socket(hostname, RedMan.port);
                SSLSocket so = (SSLSocket) Factory.createSocket(hostname, RedMan.port);
                so.setTcpNoDelay(true);
                trace.info(name + ": connected to remote RedMan on host " + hostname); 
            
                // Get I/O stream to talk to the server
                fromRedMan  = new BufferedReader(new InputStreamReader(so.getInputStream()));
                OutputStream sout = so.getOutputStream();
                toRedMan = new PrintWriter(new OutputStreamWriter(sout));
                if (checkWithRedMan(fromRedMan, toRedMan)) return(toRedMan);
                else {
                    trace.info("Authentication with remote RedMan failed");
                    return(null);
                }
            }
            catch (Exception e) {
                trace.error(name + " - Exception: " + e.getMessage());
                return(null);
            }
        }
                     
        public void run() {     // RemRedManCon
            trace.debug(name + ": RemRedManCon thread for application is running");
            
            // Extract name of remote host
            StringTokenizer st = new StringTokenizer(remotehosts);
            String remotehost = st.nextToken();
            
            // Attemp to connect
            for (int i = 1; i <= 10; i++ ) {
                try {
                    trace.debug(name + ": RedMan trying to connect to remote RedMan");

                    // Connect to remote Redundancy Manager
                    PrintWriter outStream = connectToRemoteRedMan(remotehost);
                    if (outStream != null) {
                        outToRemoteRedMan = outStream;
                        trace.info(name + ": on host " + localhost 
                                        + ", successfully connected to remote RedMan on host "
                                        + remotehost);
                        String remotestate = engine.getRemoteState();
                        
                        break;
                    }
                    sleep(1000);
                }
                catch (Exception e) {
                    trace.error(name + " - Exception: " + e.getMessage());
                }
            }
            
            // Send current state message if connected
            if (outToRemoteRedMan != null) {
                firstmsgtoremote = true;
                trace.debug(name + ": connected to remote RedMan. Will send state to remote app.");
                String state = engine.getState();
                if (state.equals("STANDBY") && split) {
                    split = false;
                    trace.debug("SPLIT BRAIN BEING RESOLVED - Will send extra 'RAPP ACTIVE' event to remote RedMan");
                    sendStateToRemoteRedMan("ACTIVE");
                }
                sendStateToRemoteRedMan(state);
            }
            else {
                notifyAppOfRemStateChange("UNKNOWN COMM");
                trace.warn(name + ": was not able to connect to remote RedMan. Thread exiting ...");
            }    
            
            // Set reference to null so that this thread object will be garbage collected
            remredmancon = null;
        } 
    }
    
    public boolean appIsDown() {
        for (int i=0; i < compnum; i++) {  
            trace.debug("Will check if component " + compmons[i].name + " is down");
            if (!compmons[i].componentIsDown()) {
                trace.debug("Component " + name + ":" + compmons[i].name 
                                                + " is UP, so the application is still UP");
                return(false);
            }
            else {
                trace.debug("Component " + name + ":" + compmons[i].name + " is DOWN");
            }
        }
        return(true);
    }
        
    //
    // This method executes one script that shuts down all local components
    // of the application.
    //
    // If the application level shutdown script was not specified it 
    // forcefully kills the application, one component at a time.
    //
    // Similarly, if there is an exception during the execution of the
    // script, it also forcefully kills the application, one component at a
    // time.
    //
    private void appOneScriptCleanup()  {
        trace.debug(name + ":" + " one script shutdownpath = " + shutdownpath);
        if (shutdownpath != null) {
            appkilled = false;
        
            // Schedule task to kill app after shutdown time is elapsed
            TimerTask appKillerTask = new TimerTask() {
                public void run() {  appKill("SHUTDOWN"); }
            };  
            Timer timer = new Timer();
            timer.schedule(appKillerTask, shutdowntime);
            
            try {
                trace.info("RedMan will execute " + shutdownpath + " to shutdown " + name);
                Runtime.getRuntime().exec(shutdownpath);
            }
            catch (Exception e) {
                trace.error(name + " - Exception: " + e.getMessage()
                             + " - RedMan failed to execute app shutdown script. "
                             + " App processes will be forcefully killed");
                appKill("SHUTDOWN");
            }
            if (!appkilled) appShutdownWait();
            if (!appkilled) {
                appKillerTask.cancel();
                trace.debug(name + ": application shutdown within shutdown time - appKillerTask cancelled");
                for (int i=0; i < compnum; i++) compmons[i].looseEndsCleaning(false);
            }
        }
        else {
            trace.info(name + ": RedMan will forcefully kill app processes");
            appKill("SHUTDOWN");
        }        
    }
    
    boolean appkilled = false;
    private void appKill(String cause) {
        trace.error(name + ": RedMan will forcefully kill app processes due to a " + cause.toLowerCase());
        for (int i=0; i < compnum; i++) {            
            compmons[i].failureHandled = true;
            compmons[i].cleanupComponent(true);
        }
        appkilled = true;
    }
    
    // This method cleans up an application one component at a time,
    // by running one script per component to gracefully (hopefully) 
    // shutdown the component.
    private void appGracefulShutdown() {
        appkilled = false;
        
        // Schedule task to kill app after shutdown time is elapsed
        TimerTask appKillerTask = new TimerTask() {
            public void run() {  appKill("SHUTDOWN"); }
        };  
        Timer timer = new Timer();
        timer.schedule(appKillerTask, shutdowntime);
        
        // Proceed to perform a graceful shutdown
        trace.debug(name 
                + ": RedMan will kill each app process gracefully if shutdown scripts were specified");
        trace.debug("appKillerTask scheduled");
        for (int i=(compnum-1); i >= 0; i--) {    // Shutdown in the reverse order of startup            
            shutdownCompAndWait(i);
            if (appkilled) {
                trace.debug("App was killed by appKillerTask - Was not shutdown within shutdown time");
                break;
            }
        }
        
        if (!appkilled) {
            appKillerTask.cancel();
            trace.debug(name + ": application shutdown within shutdown time - appKillerTask cancelled");
            for (int i=0; i < compnum; i++) compmons[i].looseEndsCleaning(false);
        }

    }
    
    private void shutdownCompAndWait(int index) {
        compmons[index].failureHandled = true;
        compmons[index].cleanupComponent(false);
        
        boolean compdown = false;
        do {
            try {
                Thread.sleep(1000);
            }
            catch (InterruptedException e) {
                trace.warn("shutdownCompAndWait() sleep interrupted");
            }
            if (compmons[index].componentIsDown() && !appkilled) {
                trace.debug(name + ":" + compmons[index].name 
                                 + " component shutdown before the graceful shutdown time elapsed");
                compdown = true;
            }
        } while ((compdown == false) && !appkilled);
        if (appkilled) trace.debug(name + " was killed - Exiting shutdownCompAndWait()");
    }
    
    private void appShutdownWait() {
        boolean appdown = false;
        do {
            try {
                Thread.sleep(3000);
            }
            catch (InterruptedException e) {
                trace.warn("appShutdownWait() sleep interrupted");
            }
            if (appIsDown() && !appkilled) {
                trace.debug(name + ": is down before the graceful shutdown time elapsed");
                appdown = true;
            }
        } while ((appdown == false) && !appkilled);
        if (appkilled) trace.debug(name + " was killed - Exiting appShutdownWait()");
    }  
    
    //
    // This method is executed as a result of an operator command that
    // causes the application to be brough down (DISABLE, SHUTDOWN, or
    // INIT).
    //
    // The local components of the application are shutdown in three 
    // possible ways:
    // (a) By sending an "OFFLINE" state change message to each process;
    // (b) By running a script that shuts down all the local components
    //     of the application;
    // (c) By shutting down one component at a time: either running 
    //     a script that shuts the component down, or by forcefully
    //     killing the processes of the component.
    //
    private void appShutdown() {
        String state = engine.getState();
        int i;
        for (i=0; i < compnum; i++) compmons[i].failureHandled = true;
        if (local) {
            submitHardEventToSMA("APP OFFLINE");
            
            // Propagate the state change to the backup RedMan
            if (RedMan.backupwriter != null) {
                RedMan.backupwriter.println(name + " STATE OFFLINE");
                RedMan.backupwriter.flush();
                trace.debug("PRIMARY RedMan sent state msg to BACKUP RedMan: '" 
                                                + name + " STATE OFFLINE'");
            }
            if (restartinprogress) sendStateToRemoteRedMan("OFFLINE RESTART");
            else sendStateToRemoteRedMan("OFFLINE");
        }
        if (shutdownpath != null) {
            if (shutdownpath.equals("*")) {
                trace.info("RedMan will command " + name + " to change state from " 
                                                         + state + " to OFFLINE");
                for (i = 0; i < compnum; i++)
                    if (compmons[i].local) compmons[i].notifyLocalCompOfStateChange("OFFLINE");
            }
            else appOneScriptCleanup();
        }
        else appGracefulShutdown();
    }
    
    private void appFailShutdown(String cause) {
        trace.debug("Executing appFailShutdown(" + cause + ")");
        // If the application is starting, cancel the startup thread and wait until 
        // that thread has indeed stopped executing
        if (starting) {
            trace.debug("AppFailShutdown() called but app still starting");
            int times = 0;
            if (starter != null) starter.cancel();
            while (starting) {
                times++;
                if (times > 10000) {
                    if (starter != null) starter.interrupt();
                    break;
                }
                try {
                    trace.debug("Will wait 2 sec for starter thread stop executing");
                    Thread.sleep(2000);
                }
                catch (InterruptedException e) {
                    trace.warn("Sleep interrupted in appFailShutdown() while waiting for app start to end");
                }
            }
        }
        
        for (int i=0; i < compnum; i++) compmons[i].failureHandled = true;
        
        if (local) {
            trace.debug("AppFailShutdown() will submit \"APP FAILED\" event to SMA");
            submitHardEventToSMA("APP FAILED");
            sendStateToRemoteRedMan("FAILED " + cause);
            notifyDistCompOfStateChange("FAILED " + cause);
        }
        else trace.debug("appFailShutdown() - App " + name + " not local");
        appKill("FAILURE");
 
        if (!local) notifyApplicationMonitor("FAILED " + cause);
    }
    
    public synchronized String makeLocalStandby() {
        String state = submitEventToSMA("APP STANDBY");
        if (state.equals("STANDBY")) return("OK");
        else return("Unable to SWITCH application");
    }
    
    public synchronized String makeRemoteStandby() {
        sendCommandToRemoteRedMan("SWITCH");
        return("OK");
    }
    
    public synchronized String makeLocalActive() {
        String state = submitEventToSMA("APP ACTIVE");
        if (state.equals("ACTIVE")) return("OK");
        else return("Unable to SWITCH application");
    }   
    
    public synchronized String makeRemoteActive() {
        sendCommandToRemoteRedMan("SWITCH");
        return("OK");
    }   
    
    public synchronized String attemptToDriveAppToMaintenance() { 
        String state = submitEventToSMA("APP TEST");
        boolean res;
        if ((state != null) && state.equals("MAINTENANCE")) {
            failoverenabled = false;
            if (!hamodel.equals("1")) {
                inhibitFailoverRemote();
                notifyAppOfLocalEvent("INHIBITED");
                engine.setFailoverEnabled(false);
                res = writeDynamicConfigFile("FAILOVER_ENABLED", "FALSE");
                if (res == false)
                    trace.warn("Unable to write FAILOVER_ENABLED=FALSE in " 
                                                        + name + " dynamic config file");
            }
            else {                
                engine.setFailoverEnabled(false);
                res = writeDynamicConfigFile("MAINTENANCE_STATE", "TRUE");               
                if (res == false)
                    trace.warn("Unable to write MAINTENANCE_STATE=TRUE in " 
                                                        + name + " dynamic config file");          
            }
            return("OK");
        }
        else {
            if (state == null) {
                if (hamodel.equals("1")) 
                    return(name + " is not in state ACTIVE");
                else if (hamodel.equals("1+1"))
                    return(name + " is not in state STANDBY");
            }
            else return(name + " erroneously changed state to " + state);
        }
        return("OK");
    }
    
    public synchronized String attemptToReleaseAppFromMaintenance() {
        String state = submitEventToSMA("APP UNTEST");
        boolean res;
        if ((state != null) && (state.equals("STANDBY") || state.startsWith("ACTIVE"))) {
            failoverenabled = true;
            if (!hamodel.equals("1")) {
                uninhibitFailoverRemote();
                notifyAppOfLocalEvent("UNINHIBITED");
                engine.setFailoverEnabled(true);
                res = writeDynamicConfigFile("FAILOVER_ENABLED", "TRUE");
                if (res == false) 
                    trace.warn("Unable to write FAILOVER_ENABLED=TRUE in " 
                                                        + name + " dynamic config file");
            }
            else {
                engine.setFailoverEnabled(true);
                res = writeDynamicConfigFile("MAINTENANCE_STATE", "FALSE");
                if (res == false)
                    trace.warn("Unable to write MAINTENANCE_STATE=FALSE in " 
                                                        + name + " dynamic config file");                  
            }
            if (state.equals("STANDBY")) return("OK STANDBY");
            else return("OK ACTIVE");
        }
        else {
            if (state == null) return(name + " is not in state MAINTENANCE");
            else return(name + " erroneously changed state to " + state);
        }
    }
    
    public synchronized String setNewPrimaryHost(String host, boolean badprimaryevent) {
        trace.debug("Host " + host + " requested to be set as PRIMARY");
        if (host.equals(localhost) && designation) return("OK");
        if (host.equals(remotehost) && !designation) return("OK");
        boolean res = writeDynamicConfigFile("PRIMARY", host);
        if (res) {
            if (host.equals(localhost)) designation = true;
            else designation = false;
            if (engine != null) engine.setDesignation(designation);
            notifyAppOfConfigChange("DESIGNATION");
            if (badprimaryevent) notifyAppOfLocalEvent("BAD_DESIGNATION");
            return("OK");
        }
        else return("Unable to write to dynamic config file");
    }
    
    public synchronized String inhibitFailoverLocal() {
        if (!failoverenabled) return("OK");
        boolean res = writeDynamicConfigFile("FAILOVER_ENABLED", "FALSE");        
        if (res) {
            trace.debug("Wrote FAILOVER_ENABLED=FALSE in dynamic config file");
            failoverenabled = false;
            engine.setFailoverEnabled(false);
            notifyAppOfLocalEvent("INHIBITED");
            return("OK");
        }
        else return("Unable to write to dynamic config file");
    }
    
    public synchronized String inhibitFailoverRemote() {
        sendCommandToRemoteRedMan("INHIBIT");
        return("OK");
    }
    
    public synchronized String uninhibitFailoverLocal() {
        if (failoverenabled) return("OK");
        boolean res = writeDynamicConfigFile("FAILOVER_ENABLED", "TRUE");
        if (res) {
            notifyAppOfLocalEvent("UNINHIBITED");
            String state, newstate;
            state = engine.getState();
            failoverenabled = true;
            newstate = engine.setFailoverEnabled(true);            
            if (state.equals("STANDBY") && newstate != null && newstate.equals("ACTIVE")) {
                notifyAppAndBackupRedManOfStateChange(newstate + " UNINHIBIT");
            }
            return("OK");
        }
        else return("Unable to write to dynamic config file");
    }
    
    public synchronized String uninhibitFailoverRemote() {
        sendCommandToRemoteRedMan("UNINHIBIT");
        return("OK");
    }
    
    public String handleShutdown(String cmd) {
        if (!cmd.equals("CFGFILE")) {
            appShutdown();
            running = false;
            trace.debug("App " + name + " is shutdown because of a " + cmd + " command");
        }
        if (!hamodel.equals("1")) stopHostMon();
        if (cmd.equals("DISABLE") || cmd.equals("CFGFILE")) {
            if (writeDynamicConfigFile("START_ENABLED", "FALSE")) return("OK");
            else return("Unable to write to application config file");
        } 
        return("OK");
    }
    
    public synchronized String handleStart(String cmd) {
        if (!cmd.equals("CFGFILE")) {
            if (appCheckRunAndStart()) {
                running = true;
                trace.debug("App " + name + " is started because of a " + cmd + " command");
            }
            else {
                return("Unable to start application, startup either failed or not configured");
            }
        }
        if (cmd.equals("ENABLE") || cmd.equals("CFGFILE")) {
            if (writeDynamicConfigFile("START_ENABLED", "TRUE")) return("OK");
            else return("Unable to write to application config file");
        }    
        return("OK");
    }
   
    private boolean failureThresholdExceeded() {        
        if (threshold == 0 ) return(true);
        t2 = System.currentTimeMillis();
        numfault++;
        if (t1 == 0) {
            t1 = t2;            
            return(false);
        }
        else if ((t2 - t1) <= timewindow) {
                trace.debug("Calculating failure threshold, numfault = " + String.valueOf(numfault));
                if (numfault > threshold) {
                    t1 = 0;
                    return(true);
                }
                else return(false);
             }
        else {
            numfault = 1;
            trace.debug("Calculating failure threshold, numfault = " + String.valueOf(numfault));
            t1 = t2;
            return(false);
        }
    } 
    
    public void handleFailure(String cause) {
        trace.debug("Executing handleFailure(" + cause + ")");
        trace.debug("App " + name + " failed");
        
        appFailShutdown(cause);
        running = false;
        trace.debug("App " + name + " is shutdown because of a failure");

        if (hamodel.equals("1+0")) {
            if (!failureThresholdExceeded()) {
                if (engine.getState().equals("ACTIVE")) {                
                    appStart(name, true);
                    running = true;
                    trace.debug("App " + name + " has been restarted");
                }
            }
            else trace.debug("App " + name + " won't be restarted - Failure threshold exceeded");
        }
        
        if (hamodel.equals("1+1") || hamodel.equals("1")) {
            if (!failureThresholdExceeded()) {
                trace.debug("App " + name + " will be restarted - Failure threshold not exceeded");
                appStart(name, true);
                running = true;
                
            }
            else trace.debug("App " + name + " won't be restarted - Failure threshold exceeded");

            // Set stateChgTask to null and cancel it
            if (stateChgTask != null) {
                TimerTask task = stateChgTask;
                stateChgTask = null;
                task.cancel();
            }
        }
    }
        
    public void handleDistFailure(String cause) {
        trace.debug("App " + name + " (distributed portion running on host " 
                                  + localhost + ") failed");
        appFailShutdown(cause);
        running = false;
        trace.debug("App " + name + " (distributed portion running on host " 
                                  + localhost + ") is shutdown");

        
        if (hamodel.equals("1+1") || hamodel.equals("1")) {
            appStart(name, true);
            trace.debug("App " + name + " (remote portion running on host " 
                                  + localhost + ") has been restarted");
        
            // Set stateChgTask to null and cancel it
            if (stateChgTask != null) {
                TimerTask task = stateChgTask;
                stateChgTask = null;
                task.cancel();
            }
        }
    }
        
    //
    // This method notifies the application monitor which runs on another host
    // (together with the SMA) about a state change in the portion of the app
    // that runs distributed. This change should be a failure.
    public void notifyApplicationMonitor(String notification) {
        String msg = "PRXY " + name + " " + localhost + " " + notification;
        if (outToRemoteAppMon != null) {
            trace.debug("Will send msg \""  + msg + "\""
                            + " to application monitor (for dist app " + name
                            + " on " + localhost + ") running on remote host " 
                            + appmoncon.appmonhost); 
            outToRemoteAppMon.println(msg);
            outToRemoteAppMon.flush();
        }
        else {
            String tracemsg = "Could not send msg \""  + msg + "\"";
            tracemsg += "     to remote application monitor " + name
                                + " running on remote host " + appmoncon.appmonhost;
            tracemsg += "     because OutToRemoteAppMon is null!";
            trace.warn(tracemsg);
        }
    }
    
    public String getSMAState() {
        return(engine.getCurrentState());
    }
    
    public String getSMATargetState() {
        return(engine.getTargetState());
    }
    
    public String getSMARemoteState() {
        return(engine.getRemoteState());
    }
    
    public synchronized void setSMARemoteState(String state) {
        engine.setRemoteState(state);
    }
    
    boolean split = false;
    
    private String submitHardEventToSMA(String event) {
        trace.debug("In submitHardEventToSMA - event = " + event);
        String newstate = sendEventToSMA(event);
        trace.debug("In submitHardEventToSMA - newstate = " + newstate);
        return(newstate);
    }
    
    public synchronized String submitEventToSMA(String event) {
        return(sendEventToSMA(event));
    }
    
    private String sendEventToSMA(String event) {
        String newstate, state, remstate, pureevent;
        // The RAPP FAILURE event comes as RAPP FAILURE [ DB | FLT ]
        trace.debug("In sendEventToSMA() - event = " + event);
        if (!event.equals("APP FAILED") && !event.equals("APP OFFLINE")) {
            while (engine.intransition()) {
                try {
                    trace.info("State = " + engine.getState() + ", event = " 
                                          + event + ". Will sleep 2 sec in sendEventToSMA()");
                    Thread.sleep(2000);
                }
                catch (Exception e) {
                    trace.warn("sleep() interrupted in method sendEventToSMA(): " 
                                        + e.getMessage());
                }
            }  
        }
        if (stateChgTask != null) {
            stateChgTask.cancel();
            stateChgTask = null;
        }
            
        state = engine.getState();
        remstate = engine.getRemoteState();
        if (state.equals("ACTIVE") && event.equals("RAPP ACTIVE") && !hamodupgrade) split = true;
        
        // Get rid of the cause of failure if the event is "RAPP FAILED [DB|FLT]"
        if (event.startsWith("RAPP FAILED")) pureevent = "RAPP FAILED";
        else pureevent = event;
        
        // Run the event
        newstate = engine.run(pureevent);
        
        if ((newstate != null) && (newstate != "INIT") 
                               && (newstate != "OFFLINE") && (newstate != "FAILED")) {
            // Add information to the cause of a state change to ACTIVE
            if (newstate.equals("ACTIVE")) {
                if (event.equals("APP INITIALIZED")) newstate += " INIT";
                if (event.equals("APP ACTIVE")) newstate += " SWITCH";
                if (event.equals("APP UNTEST")) newstate += " UNTEST";
                if (event.startsWith("RAPP FAILED")) {
                    if (event.indexOf(" DB") >= 0) newstate += " DB";
                    else newstate += " FLT";
                }
                if (event.equals("RAPP OFFLINE")) newstate += " SHUTDOWN";
                if (event.equals("RHOST DOWN")) newstate += " COMM";
                if (event.equals("RAPP STANDBY")) {
                    if (hamodupgrade) newstate += " HA_MODEL_UPGRADE";
                    else newstate += " SPTBR_RECV";
                    if (split) split = false;
                    if (hamodupgrade) hamodupgrade = false;
                }
                if (event.equals("RAPP MAINTENANCE") 
                            && remstate.equals("INIT")) newstate += " INIT";
            }
            notifyAppAndBackupRedManOfStateChange(newstate);
            if (event.equals("APP INITIALIZED")) {
                if (remstate.equals("FAILED")) {
                    if (outToRemoteRedMan == null) remstate += " COMM";
                    else remstate += " FLT";
                }
                notifyAppOfRemStateChange(remstate);
            }
            if (badprimary) {
                notifyAppOfConfigChange("DESIGNATION");
                notifyAppOfLocalEvent("BAD_DESIGNATION");
                badprimary = false;
            }
        }
        
        return(newstate);
    }
    
    public String commitNewSMAState() {
        trace.debug(name + ": will call engine.run(COMMIT)");             
        String state = engine.run("COMMIT");
        if (state != null) {
            if (stateChgTask != null) {
                trace.debug("Will cancel stateChgTask -- state = " + state);                
                stateChgTask.cancel();
                stateChgTask = null;
            }
            if (state.equals("STANDBY") && switchinprogress) {
                makeRemoteActive();
                switchinprogress = false;
            }

            if (state.equals("STANDBY") && (split || hamodupgrade)) {                    
                if ((activesent == false) && (outToRemoteRedMan != null)) {
                    if (split) trace.debug("RESOLVING SPLIT BRAIN - Sending extra 'RAPP ACTIVE' msg to remote RedMan");
                    if (hamodupgrade) trace.debug("RESOLVING UPGRADE - Sending extra 'RAPP ACTIVE' msg to remote RedMan");
                    sendStateToRemoteRedMan("ACTIVE");
                }
                else trace.debug("Unable to send state to remote RedMan. No connection");
                activesent = false;
            }
            else activesent = false;
            if (outToRemoteRedMan != null) sendStateToRemoteRedMan(state);
            else trace.debug("Unable to send state to remote RedMan. No connection");            
        }
        return(state);
    }
    
    // This method sends state change information, due to a state change in
    // the SMA ruunning locally, to distributed components of the application 
    // running on other hosts.
    private void notifyDistCompOfStateChange(String state) {
        if (!local) return;
        String[] hosts = new String[numdistmngdmsgs];
        int index = 0;
        for (int i = 0; i < compnum; i++) {
            if (!compmons[i].local && (compmons[i].out != null)) {
                String host = compmons[i].host;
                boolean found = false;
                for (int j = 0; j < index; j++) {
                    if (host.equals(hosts[j])) {
                        found = true;
                        break;
                    }
                }
                if (!found) {
                    hosts[index] = host;
                    index++;
                    compmons[i].out.println("CMD " + state);
                    if (index >= numdistmngdmsgs) break;
                }
            }
        }
    }
    
    // This method sends event information, due to a change in  the  local
    // failover_enabled flag, to distributed components of the application 
    // running on other hosts.
    private void notifyDistCompOfLocalEvent(String event) {
        if (!local) return;
        String[] hosts = new String[numdistmngdmsgs];
        int index = 0;
        for (int i = 0; i < compnum; i++) {
            if (!compmons[i].local && (compmons[i].out != null)) {
                String host = compmons[i].host;
                boolean found = false;
                for (int j = 0; j < index; j++) {
                    if (host.equals(hosts[j])) {
                        found = true;
                        break;
                    }
                }
                if (!found) {
                    hosts[index] = host;
                    index++;
                    compmons[i].out.println("EVENT LOCAL " + event);
                    if (index >= numdistmngdmsgs) break;
                }
            }
        }
    }
    
    private void notifyDistCompOfRemStateChange(String remstate) {
        if (!local) return;
        String[] hosts = new String[numdistmngdmsgs];
        int index = 0;
        for (int i = 0; i < compnum; i++) {
            if (!compmons[i].local && (compmons[i].out != null)) {
                String host = compmons[i].host;
                boolean found = false;
                for (int j = 0; j < index; j++) {
                    if (host.equals(hosts[j])) {
                        found = true;
                        break;
                    }
                }
                if (!found) {
                    hosts[index] = host;
                    index++;
                    if (compmons[i].remeventmsg) compmons[i].out.println("RAPP STATE " + remstate);
                    if (index >= numdistmngdmsgs) break;
                }
            }
        }
    }
    
    public void notifyAppOfRemStateChange(String remstate) {
        String debmsg = name + ": will only send rem state ";
        debmsg += "change msg to comp/subcomp that has requested it";
        trace.debug(debmsg );
        
        for (int i = 0; i < compnum; i++) {
            if (compmons[i].local && compmons[i].remeventmsg) 
                compmons[i].notifyLocalCompOfRemStateChange(remstate);
        }
        
        notifyDistCompOfRemStateChange(remstate);
    }
    
    private void scheduleStateChgTask() {
        // Schedule task that will cause failure if state change 
        // does not happen within configured time
        if (local) {
            stateChgTask = new TimerTask() {
                public void run() {  
                    trace.error("scheduleStateChgTask(): state change timed out - Will fail app " + name);
                    handleFailure("FLT"); 
                }
            };
        }
        else {
            stateChgTask = new TimerTask() {
                public void run() {  handleDistFailure("FLT"); }
            }; 
        }
        Timer timer = new Timer();
        timer.schedule(stateChgTask, statechgtime);        
    }
    
    
    private void notifyAppAndBackupRedManOfStateChange(String state) {
        
        scheduleStateChgTask();
        
        trace.debug("Will notify app " + name + " of state change to " + state);
        for (int i = 0; i < compnum; i++) {
            if (compmons[i].local) compmons[i].notifyLocalCompOfStateChange(state);
        }
        
        notifyDistCompOfStateChange(state);
        
        // Propagate the state change to the backup RedMan
        if (RedMan.backupwriter != null) {
            RedMan.backupwriter.println(name + " STATE " + state);
            RedMan.backupwriter.flush();
            trace.debug("PRIMARY RedMan sent state msg to BACKUP RedMan: '" 
                                                + name + " STATE " + state + "'");
        }
    }
    
    public void notifyAppOfLocalEvent(String event) {
        trace.debug("Will notify app " + name + " of local event " + event);
        for (int i = 0; i < compnum; i++) {
            if (compmons[i].local) compmons[i].notifyLocalCompOfLocalEvent(event);
        }
        
        notifyDistCompOfLocalEvent(event);
    }
    
    public void notifyAppOfConfigChange(String change) {
        trace.debug("Will notify app " + name + " of config change " + change);
        for (int i = 0; i < compnum; i++) {
            if (compmons[i].local) compmons[i].notifyLocalCompOfConfigChange(change);
        }
        
        // notifyDistCompOfConfigChange(change);
    }
    
    public int getProcPid(String procname) {
        int pid = -1;
        for (int i = 0; i < compnum; i++) {
            pid = compmons[i].getProcPid(procname);
            if (pid != -1) break;
        }
        return(pid);
    }
    
    public void handleHandShake()  {
        trace.debug("Got to handleHandShake()");
        if (waitingforhandshake) {
            trace.debug("Thread was waiting for handshake...");
            stopWaitingForHandShake();
        }
        else {
            trace.debug("Thread was NOT waiting for handshake - Will send handshake to peer RedMan");
            if (outToRemoteRedMan != null) {
                try {
                    outToRemoteRedMan.println("RAPP HANDSHAKE " + name + " " + localhost);
                    outToRemoteRedMan.flush();
                    trace.debug("Sent handshake message to RedMan on host " + remotehost);
                }
                catch (Exception e) {
                    trace.debug("Could not send handshake to peer RedMan - " + e.getMessage());
                    return;
                }
            }
            else {
                trace.debug("Could not send handshake to peer RedMan - no connection");
                return;
            }
        }
    }
        
    public boolean waitForHandShake() {
        trace.debug("This thread will wait for handshake...");
        synchronized (handshake) {
            waitingforhandshake = true;
        }
        int counter = 0;
        do {
            try {
                Thread.sleep(100);
                counter++;
                if (counter == 50) {
                    trace.debug("Got out of wait-for-handshake loop - Handshake not received (timed out)");
                    return(false);
                }
            }
            catch (Exception e) {
                trace.debug("Exception during Thread.sleep(): " + e.getMessage());
            }
        } while (waitingforhandshake);
        trace.debug("Got out of wait-for-handshake loop - Handshake received");
        return(true);
    }
            
    public void stopWaitingForHandShake() {
        synchronized (handshake) {
            waitingforhandshake = false;
        }
    }  
        
    public boolean isCommunicatingWithPeer() {
        if (outToRemoteRedMan == null) return(false);           
        try {
            synchronized (handshake) {
                outToRemoteRedMan.println("RAPP HANDSHAKE " + name + " " + localhost);
                outToRemoteRedMan.flush();
            }
            trace.debug("Initiating communication check. Sent handshake message to peer RedMan");
        }
        catch (Exception e) {
            trace.debug("Could not send handshake to peer RedMan - " + e.getMessage());
            return(false);
        }
        if (waitForHandShake()) {
            trace.debug("Handshake received from peer RedMan");
            return(true);
        }
        else {
            trace.debug("Handshake not received from peer RedMan (timed out)");
            return(false);
        }
    }
}